{
  "contentVersion": "1.0.0.0",
  "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
  "parameters": {
    "storageAccountName": {
      "type": "string",
      "metadata": {
        "description": "Name of the Azure storage account that contains the input/output data."
      }
    },
    "storageAccountKey": {
      "type": "securestring",
      "metadata": {
        "description": "Key for the Azure storage account."
      }
    },
    "batchAccountName": {
      "type": "string",
      "metadata": {
        "description": "Azure Batch account name."
      }
    },
    "batchAccessKey": {
      "type": "securestring",
      "metadata": {
        "description": "Azure Batch account access key."
      }
    },
    "batchPoolName": {
      "type": "string",
      "metadata": {
        "description": "Azure Batch account pool name."
      }
    },
    "batchURL": {
      "type": "string",
      "metadata": {
        "description": "Azure Batch account URL."
      }
    },
    "sfdcUserName": {
      "type": "string",
      "metadata": {
        "description": "SFDC User Name."
      }
    },
    "sfdcPassword": {
      "type": "string",
      "metadata": {
        "description": "SFDC password."
      }
    },
    "sfdcSecurityToken": {
      "type": "securestring",
      "metadata": {
        "description": "Security Token for SFDC instance."
      }
    },
    "blobContainer": {
      "type": "string",
      "metadata": {
        "description": "Name of the blob container in the Azure Storage account."
      }
    },
    "runFrequency": {
      "type": "string",
      "metadata": {
        "description": "ADF run frequency."
      }
    },
    "runInterval": {
      "type": "string",
      "metadata": {
        "description": "ADF run interval."
      }
    },
    "freqStartDate": {
      "type": "string",
      "metadata": {
        "description": "ADF frequency Start Date."
      }
    },
    "freqEndDate": {
      "type": "string",
      "metadata": {
        "description": "ADF frequency End Date."
      }
    },
    "geoLocation": {
      "type": "string",
      "metadata": {
        "description": "Geo-Location where the resource group needs to be deployed."
      }
    },
    "cognitiveUserId": {
      "type": "string",
      "metadata": {
        "description": "Registered Cognitive Services User Id."
      }
    },
    "cognitiveSubscriptionId": {
      "type": "string",
      "metadata": {
        "description": "Subscription Id for Cognitive Services Recommendation API."
      }
    },
    "ADFName": {
      "type": "string",
      "metadata": {
        "description": "Name of  Azure Data Factory."
      }
    }
  },
  "variables": {
    "dataFactoryName": "[parameters('ADFName')]",
    "azureStorageLinkedServiceName": "ls_AzStorage",
    "azureBatchLinkedServiceName": "ls_AzureBatch",
    "sfdcLinkedServiceName": "ls_SFDC",
    "blobCatalogSinkDatasetName": "ds_BlobSinkCatalog",
    "blobUsageSinkDatasetName": "ds_BlobSinkUsage",
    "blobOutputTrainedDataDatasetName": "ds_OutputTrainedData",
    "blobInputOppLineItems": "ds_OppLineItems",
    "blobInputPriceBook2": "ds_PriceBook2",
    "blobInputPricebookEntry2": "ds_PriceBookEntry2",
    "sfdcOppLineItemDatasetName": "ds_SFDC_OppLineItem",
    "sfdcPriceBook2DatasetName": "ds_SFDC_Pricebook2",
    "sfdcPriceBookEntry2DatasetName": "ds_SFDC_PricebookEntry",
    "generateCatalogData": "pl_GenerateCatalogData",
    "TraininPipelineName": "pl_RecoTraining",
    "SFDCtoBlobSinkPipelineName": "pl_SFDCToBlobSink",
    "getSFDCDetails" : "pl_SFDCDetails"
  },
  "resources": [
    {
      "name": "[variables('dataFactoryName')]",
      "apiVersion": "2015-10-01",
      "type": "Microsoft.DataFactory/datafactories",
      "location": "[parameters('geoLocation')]",
      "resources": [
        {
          "type": "linkedservices",
          "name": "[variables('azureStorageLinkedServiceName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "AzureStorage",
            "description": "Azure Storage linked service",
            "typeProperties": {
              "connectionString": "[concat('DefaultEndpointsProtocol=https;AccountName=',parameters('storageAccountName'),';AccountKey=',parameters('storageAccountKey'))]"
            }
          }
        },
        {
          "type": "linkedservices",
          "name": "[variables('azureBatchLinkedServiceName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "AzureBatch",
            "typeProperties": {
              "accountName": "[parameters('batchAccountName')]",
              "accessKey": "[parameters('batchAccessKey')]",
              "poolName": "[parameters('batchPoolName')]",
              "batchUri": "[parameters('batchURL')]",
              "linkedServiceName": "[variables('azureStorageLinkedServiceName')]"
            }
          }
        },
        {
          "type": "linkedservices",
          "name": "[variables('sfdcLinkedServiceName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "Salesforce",
            "typeProperties": {
              "username": "[parameters('sfdcUserName')]",
              "password": "[parameters('sfdcPassword')]",
              "securityToken": "[parameters('sfdcSecurityToken')]"
            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('sfdcOppLineItemDatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('sfdcLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "RelationalTable",
            "linkedServiceName": "[variables('sfdcLinkedServiceName')]",
            "structure": [
              {
                "name": "AccountId",
                "type": "String"
              },
              {
                "name": "Product2Id",
                "type": "String"
              },
              {
                "name": "CreatedDate",
                "type": "Datetime"
              }
            ],
            "published": false,
            "typeProperties": {
              "tableName": "OpportunityLineItem"
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            },
            "external": true,
            "policy": {

            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('sfdcPriceBook2DatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('sfdcLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "RelationalTable",
            "linkedServiceName": "[variables('sfdcLinkedServiceName')]",
            "structure": [
              {
                "name": "Id",
                "type": "String"
              },
              {
                "name": "Name",
                "type": "String"
              }
            ],
            "published": false,
            "typeProperties": {
              "tableName": "Pricebook2"
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            },
            "external": true,
            "policy": {

            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('sfdcPriceBookEntry2DatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('sfdcLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "RelationalTable",
            "linkedServiceName": "[variables('sfdcLinkedServiceName')]",
            "structure": [
              {
                "name": "Pricebook2Id",
                "type": "String"
              },
              {
                "name": "Product2Id",
                "type": "String"
              },
              {
                "name": "Name",
                "type": "String"
              }
            ],
            "published": false,
            "typeProperties": {
              "tableName": "PricebookEntry"
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            },
            "external": true,
            "policy": {

            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('blobUsageSinkDatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "AzureBlob",
            "linkedServiceName": "[variables('azureStorageLinkedServiceName')]",
            "published": false,
            "typeProperties": {
              "fileName": "UsageData.csv",
              "folderPath": "[parameters('blobContainer')]",
              "format": {
                "type": "TextFormat",
                "rowDelimiter": "\n",
                "columnDelimiter": ","
              }
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('blobCatalogSinkDatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "AzureBlob",
            "linkedServiceName": "[variables('azureStorageLinkedServiceName')]",
            "typeProperties": {
              "fileName": "CatalogData.csv",
              "folderPath": "[parameters('blobContainer')]",
              "format": {
                "type": "TextFormat",
                "rowDelimiter": "\n",
                "columnDelimiter": ","
              }
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('blobOutputTrainedDataDatasetName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "type": "AzureBlob",
            "linkedServiceName": "[variables('azureStorageLinkedServiceName')]",
            "published": false,
            "typeProperties": {
              "fileName": "TrainResult.csv",
              "folderPath": "[parameters('blobContainer')]"
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            }
          }
        },        
        {
          "type": "datasets",
          "name": "[variables('blobInputPriceBook2')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "structure": [
              {
                "name": "Id",
                "type": "String"
              },
              {
                "name": "Name",
                "type": "String"
              }
            ],
            "type": "AzureBlob",
            "linkedServiceName": "[variables('azureStorageLinkedServiceName')]",
            "published": false,
            "typeProperties": {
              "fileName": "PriceBook.csv",
              "folderPath": "[parameters('blobContainer')]",
              "format": {
                "type": "TextFormat",
                "rowDelimiter": "\n",
                "columnDelimiter": ",",
                "firstRowAsHeader": true
              }
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            }
          }
        },
        {
          "type": "datasets",
          "name": "[variables('blobInputPricebookEntry2')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "structure": [
              {
                "name": "Pricebook2Id",
                "type": "String"
              },
              {
                "name": "Product2Id",
                "type": "String"
              },
              {
                "name": "Name",
                "type": "String"
              }
            ],
            "type": "AzureBlob",
            "linkedServiceName": "[variables('azureStorageLinkedServiceName')]",
            "published": false,
            "typeProperties": {
              "fileName": "PriceBookEntry.csv",
              "folderPath": "[parameters('blobContainer')]",
              "format": {
                "type": "TextFormat",
                "rowDelimiter": "\n",
                "columnDelimiter": ",",
                "firstRowAsHeader": true
              }
            },
            "availability": {
              "frequency": "[parameters('runFrequency')]",
              "interval": "[int(parameters('runInterval'))]",
              "style": "StartOfInterval"
            }
          }
        },
        {
          "type": "datapipelines",
          "name": "[variables('SFDCtoBlobSinkPipelineName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('sfdcLinkedServiceName')]",
            "[variables('sfdcOppLineItemDatasetName')]",
            "[variables('sfdcPriceBook2DatasetName')]",
            "[variables('sfdcPriceBookEntry2DatasetName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "description": "Copy catalog data from SFDC Source to Az Blob Sink.",
            "activities": [
              {
                "type": "Copy",
                "typeProperties": {
                  "source": {
                    "type": "RelationalSource",
                    "query": "$$Text.Format('select opportunity.accountid,product2ID,opportunity.createddate from OpportunityLineItem')"
                  },
                  "sink": {
                    "type": "BlobSink",
                    "writeBatchSize": 0,
                    "writeBatchTimeout": "00:00:00"
                  }
                },
                "inputs": [
                  {
                    "name": "[variables('sfdcOppLineItemDatasetName')]"
                  }
                ],
                "outputs": [
                  {
                    "name": "[variables('blobUsageSinkDatasetName')]"
                  }
                ],
                "policy": {
                  "timeout": "1.00:00:00",
                  "concurrency": 1,
                  "executionPriorityOrder": "NewestFirst",
                  "style": "StartOfInterval",
                  "retry": 3,
                  "longRetry": 0,
                  "longRetryInterval": "00:00:00"
                },
                "scheduler": {
                  "frequency": "[parameters('runFrequency')]",
                  "interval": "[int(parameters('runInterval'))]",
                  "style": "StartOfInterval"
                },
                "name": "Copy0"
              },
              {
                "type": "Copy",
                "typeProperties": {
                  "source": {
                    "type": "RelationalSource",
                    "query": "$$Text.Format('SELECT Id,Name FROM Pricebook2')"
                  },
                  "sink": {
                    "type": "BlobSink",
                    "writeBatchSize": 0,
                    "writeBatchTimeout": "00:00:00"
                  }
                },
                "inputs": [
                  {
                    "name": "[variables('sfdcPriceBook2DatasetName')]"
                  }
                ],
                "outputs": [
                  {
                    "name": "[variables('blobInputPriceBook2')]"
                  }
                ],
                "policy": {
                  "timeout": "1.00:00:00",
                  "concurrency": 1,
                  "executionPriorityOrder": "NewestFirst",
                  "style": "StartOfInterval",
                  "retry": 3,
                  "longRetry": 0,
                  "longRetryInterval": "00:00:00"
                },
                "scheduler": {
                  "frequency": "[parameters('runFrequency')]",
                  "interval": "[int(parameters('runInterval'))]",
                  "style": "StartOfInterval"
                },
                "name": "Copy1"
              },
              {
                "type": "Copy",
                "typeProperties": {
                  "source": {
                    "type": "RelationalSource",
                    "query": "$$Text.Format('SELECT Pricebook2Id,Product2Id,Product2.Name FROM PricebookEntry')"
                  },
                  "sink": {
                    "type": "BlobSink",
                    "writeBatchSize": 0,
                    "writeBatchTimeout": "00:00:00"
                  }
                },
                "inputs": [
                  {
                    "name": "[variables('sfdcPriceBookEntry2DatasetName')]"
                  }
                ],
                "outputs": [
                  {
                    "name": "[variables('blobInputPricebookEntry2')]"
                  }
                ],
                "policy": {
                  "timeout": "1.00:00:00",
                  "concurrency": 1,
                  "executionPriorityOrder": "NewestFirst",
                  "style": "StartOfInterval",
                  "retry": 3,
                  "longRetry": 0,
                  "longRetryInterval": "00:00:00"
                },
                "scheduler": {
                  "frequency": "[parameters('runFrequency')]",
                  "interval": "[int(parameters('runInterval'))]",
                  "style": "StartOfInterval"
                },
                "name": "Copy2"
              }
            ],
            "start": "[parameters('freqStartDate')]",
            "end": "[parameters('freqEndDate')]",
            "isPaused": false
          }
        },
        {
          "type": "datapipelines",
          "name": "[variables('generateCatalogData')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]",
            "[variables('blobInputPricebookEntry2')]",
            "[variables('blobInputPriceBook2')]",
            "[variables('blobCatalogSinkDatasetName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "description": "Runs a custom activity and merges the blobs PriceBookEntry and PriceBook.",
            "activities": [
              {
                "type": "DotNetActivity",
                "typeProperties": {
                  "assemblyName": "CatalogDataActivity.dll",
                  "entryPoint": "Microsoft.Cis.Recommendation.CatalogDataActivity.GenerateCatalogData",
                  "packageLinkedService": "[variables('azureStorageLinkedServiceName')]",
                  "packageFile": "[concat(parameters('blobContainer'),'/GenerateCatalog.zip')]",
                  "extendedProperties": {
                    "Year": "$$Text.Format('{0:yyyy}',SliceStart)",
                    "Month": "$$Text.Format('{0:%M}',SliceStart)",
                    "Day": "$$Text.Format('{0:%d}',SliceStart)"
                  }
                },
                "inputs": [
                  {
                    "name": "[variables('blobInputPriceBook2')]"
                  },
                  {
                    "name": "[variables('blobInputPricebookEntry2')]"
                  }
                ],
                "outputs": [
                  {
                    "name": "[variables('blobCatalogSinkDatasetName')]"
                  }
                ],
                "policy": {
                  "timeout": "01:00:00",
                  "concurrency": 1,
                  "retry": 3
                },
                "scheduler": {
                  "frequency": "[parameters('runFrequency')]",
                  "interval": "[int(parameters('runInterval'))]",
                  "style": "StartOfInterval"
                },
                "name": "GenerateCatalogData",
                "linkedServiceName": "[variables('azureBatchLinkedServiceName')]"
              }
            ],
            "start": "[parameters('freqStartDate')]",
            "end": "[parameters('freqEndDate')]",
            "isPaused": false
          }
        },
        {
          "type": "datapipelines",
          "name": "[variables('TraininPipelineName')]",
          "dependsOn": [
            "[variables('dataFactoryName')]",
            "[variables('azureStorageLinkedServiceName')]",
            "[variables('blobCatalogSinkDatasetName')]",
            "[variables('blobUsageSinkDatasetName')]",
            "[variables('blobOutputTrainedDataDatasetName')]"
          ],
          "apiVersion": "2015-10-01",
          "properties": {
            "description": "Uploads processed catalog and usage data to the Azure 'Recommendations' model.",
            "activities": [
              {
                "type": "DotNetActivity",
                "typeProperties": {
                  "assemblyName": "RecommendationTrainActivity.dll",
                  "entryPoint": "Microsoft.Cis.Recommendation.TrainingActivity.RecommendationTrainingActivity",
                  "packageLinkedService": "[variables('azureStorageLinkedServiceName')]",
                  "packageFile": "[concat(parameters('blobContainer'),'/Reco.zip')]",
                  "extendedProperties": {
                    "Year": "$$Text.Format('{0:yyyy}',SliceStart)",
                    "Month": "$$Text.Format('{0:%M}',SliceStart)",
                    "Day": "$$Text.Format('{0:%d}',SliceStart)",
                    "AzureRecommendationsEmailAddress": "[parameters('cognitiveUserId')]",
                    "AzureRecommendationsAccountKey": "[parameters('cognitiveSubscriptionId')]",
                    "AzureRecommendationsApplicationName": "Oppty",
                    "AzureRecommendationsModelBuildType": "Recommendation",
                    "AzureRecommendationsModelName": "SFDCRec",
                    "AzureRecommendationsMaxBuildsToKeep": "3",
                    "AzureRecommendationsTrainingDescription": "$$Text.Format('{0:yyyy-MM-dd HH:mm}', WindowStart, WindowEnd)"
                  }
                },
                "inputs": [
                  {
                    "name": "[variables('blobCatalogSinkDatasetName')]"
                  },
                  {
                    "name": "[variables('blobUsageSinkDatasetName')]"
                  },
                  {
                    "name" : "[variables('sfdcOppLineItemDatasetName')]"
                  }
                ],
                "outputs": [
                  {
                    "name": "[variables('blobOutputTrainedDataDatasetName')]"
                  }
                ],
                "policy": {
                  "timeout": "01:00:00",
                  "concurrency": 1,
                  "retry": 3
                },
                "scheduler": {
                  "frequency": "[parameters('runFrequency')]",
                  "interval": "[int(parameters('runInterval'))]",
                  "style": "StartOfInterval"
                },
                "name": "RecoAPI",
                "linkedServiceName": "[variables('azureBatchLinkedServiceName')]"
              }
            ],
            "start": "[parameters('freqStartDate')]",
            "end": "[parameters('freqEndDate')]",
            "isPaused": false
          }
        }        
      ]
    }
  ]
}